package com.hilda.HTMLParserTesting;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.Vector;

import android.content.Context;
import android.os.AsyncTask;
import android.os.Environment;

public class WebCrawler implements Runnable {
	public static final String SEARCH = "Search";
	public static final String STOP = "Stop";
	public static final String DISALLOW = "Disallow:";
	public static final int SEARCH_LIMIT = 1;
	private URL url = null;
	private Context mContext;

	public WebCrawler(Context c) {
		mContext = c;

	}

	List<?> listMatches;

	// URLs to be searched
	Vector<String> vectorToSearch;
	// URLs already searched
	Vector<String> vectorSearched;
	// URLs which match
	Vector<?> vectorMatches;

	Thread searchThread;

	public void init() {

		// initialize search data structures
		vectorToSearch = new Vector<String>();
		vectorSearched = new Vector<String>();
		vectorMatches = new Vector<Object>();

		// set default for URL access
		URLConnection.setDefaultAllowUserInteraction(false);
	}

	public void stop() {
		if (searchThread != null) {

			searchThread = null;
		}
	}

	boolean robotSafe(URL url) {
		String strHost = url.getHost();

		// form URL of the robots.txt file
		String strRobot = "http://" + strHost + "/robots.txt";
		URL urlRobot;
		try {
			urlRobot = new URL(strRobot);
		} catch (MalformedURLException e) {
			// something weird is happening, so don't trust it
			return false;
		}

		String strCommands;
		try {
			InputStream urlRobotStream = urlRobot.openStream();

			// read in entire file
			byte b[] = new byte[1000];
			int numRead = urlRobotStream.read(b);
			strCommands = new String(b, 0, numRead);
			while (numRead != -1) {
				if (Thread.currentThread() != searchThread)
					break;
				numRead = urlRobotStream.read(b);
				if (numRead != -1) {
					String newCommands = new String(b, 0, numRead);
					strCommands += newCommands;
				}
			}
			urlRobotStream.close();
		} catch (IOException e) {
			// if there is no robots.txt file, it is OK to search
			return true;
		}

		// assume that this robots.txt refers to us and
		// search for "Disallow:" commands.
		String strURL = url.getFile();
		int index = 0;
		while ((index = strCommands.indexOf(DISALLOW, index)) != -1) {
			index += DISALLOW.length();
			String strPath = strCommands.substring(index);
			StringTokenizer st = new StringTokenizer(strPath);

			if (!st.hasMoreTokens())
				break;

			String strBadPath = st.nextToken();

			// if the URL starts with a disallowed path, it is not safe
			if (strURL.indexOf(strBadPath) == 0)
				return false;
		}

		return true;
	}

	public void run() {
		String strURL = "http://www.elandroidelibre.com";
//		String strTargetType = choiceType.getSelectedItem();
		int numberSearched = 0;
		int numberFound = 0;

		if (strURL.length() == 0) {
//			setStatus("ERROR: must enter a starting URL");
			return;
		}

		// initialize search data structures
		vectorToSearch.removeAllElements();
		vectorSearched.removeAllElements();
		vectorMatches.removeAllElements();
		

		vectorToSearch.addElement(strURL);

		while ((vectorToSearch.size() > 0)
//				&& (Thread.currentThread() == searchThread)
				) {
			// get the first element from the to be searched list
			strURL = (String) vectorToSearch.elementAt(0);


			DownloadWebPageTask downloadWebPageTask = new DownloadWebPageTask();
			downloadWebPageTask.execute(strURL);
//				sasa

			numberSearched++;
			if (numberSearched >= SEARCH_LIMIT)
				break;
		}

//		if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)
//			setStatus("reached search limit of " + SEARCH_LIMIT);
//		else
//			setStatus("done");
		searchThread = null;
		// searchThread.stop();
	}

	private class DownloadWebPageTask extends AsyncTask<String, Void, String> {
		@Override
		protected String doInBackground(String... urls) {
//			URL url = null;
			try {
				url = new URL(urls[0]);
			} catch (MalformedURLException e) {
				return null;
			}

			// mark the URL as searched (we want this one way or the other)
			vectorToSearch.removeElementAt(0);
			vectorSearched.addElement(urls[0]);

			// can only search http: protocol URLs
			if (url.getProtocol().compareTo("http") != 0)
				return null;
			
			String response = "";
			// try opening the URL
			URLConnection urlConnection = null;
			try {
				urlConnection = url.openConnection();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

			urlConnection.setAllowUserInteraction(false);

			if (!isSdCardAvailable())
				return null;

			File file = new File(mContext.getExternalFilesDir(null)
					.getAbsolutePath() + "/test.html");
			OutputStream out = null;
			try {
				out = new FileOutputStream(file);
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

			InputStream urlStream = null;
			try {
				urlStream = url.openStream();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

			// search the input stream for links
			// first, read in the entire URL
			byte b[] = new byte[1000];
			int numRead = 0;
			try {
				numRead = urlStream.read(b);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			String content = new String(b, 0, numRead);
			while (numRead != -1) {
				try {
					out.write(b, 0, b.length);
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				try {
					numRead = urlStream.read(b);
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				if (numRead != -1) {
					String newContent = new String(b, 0, numRead);
					content += newContent;
				}
			}

			try {
				out.close();
				urlStream.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			

			return content;
		}

		@Override
		protected void onPostExecute(String result) {
			String lowerCaseContent = result.toLowerCase();

			int index = 0;
			while ((index = lowerCaseContent.indexOf("<a", index)) != -1) {
				if ((index = lowerCaseContent.indexOf("href", index)) == -1)
					break;
				if ((index = lowerCaseContent.indexOf("=", index)) == -1)
					break;

				if (Thread.currentThread() != searchThread)
					break;

				index++;
				String remaining = result.substring(index);

				StringTokenizer st = new StringTokenizer(remaining,
						"\t\n\r\">#");
				String strLink = st.nextToken();

				URL urlLink;
				try {
					urlLink = new URL(url, strLink);
					strLink = urlLink.toString();
				} catch (MalformedURLException e) {
					// setStatus("ERROR: bad URL " + strLink);
					continue;
				}

				// only look at http links
				if (urlLink.getProtocol().compareTo("http") != 0)
					break;

				try {
					// try opening the URL
					URLConnection urlLinkConnection = urlLink.openConnection();
					urlLinkConnection.setAllowUserInteraction(false);
					InputStream linkStream = urlLink.openStream();
					String strType = URLConnection
							.guessContentTypeFromStream(linkStream);
					linkStream.close();

					// if another page, add to the end of search list
					if (strType == null)
						break;
					if (strType.compareTo("text/html") == 0) {
						// check to see if this URL has already been
						// searched or is going to be searched
						if ((!vectorSearched.contains(strLink))
								&& (!vectorToSearch.contains(strLink))) {

							// test to make sure it is robot-safe!
							// if (robotSafe(urlLink))
							// vectorToSearch.addElement(strLink);
						}
					}

				} catch (IOException e) {
					// setStatus("ERROR: couldn't open URL " + strLink);
					continue;
				}
			}
		}
	}

	public static void main(String argv[]) {
		/*
		 * Behind a firewall set your proxy and port here!
		 */
		Properties props = new Properties(System.getProperties());
		props.put("http.proxySet", "true");
		props.put("http.proxyHost", "webcache-cup");
		props.put("http.proxyPort", "8080");

		Properties newprops = new Properties(props);
		System.setProperties(newprops);
		/**/
	}

	private boolean isSdCardAvailable() {
		boolean mExternalStorageWriteable = false;
		String state = Environment.getExternalStorageState();

		if (Environment.MEDIA_MOUNTED.equals(state)) {
			// We can read and write the media
			mExternalStorageWriteable = true;
		} else {
			// Something else is wrong. It may be one of many other states, but
			// all we need to know is we can't write
			mExternalStorageWriteable = false;
		}

		return mExternalStorageWriteable;
	}

}